from bs4 import BeautifulSoup
import requests

filename = "reuterstry.csv"
f = open(filename, "w")
headers = "Date, Title, Link\n"
f.write(headers)

url = 'https://www.reuters.com/search/news?blob=south+china+sea&sortBy=date&dateRange=all' #urls to scrape

def scraper(url):
	resp = requests.get(url) #gives response object
	soup = BeautifulSoup(resp.text, 'lxml') #search for data in a structured manner; lxml is parser library to structure data 

	articles_section = soup.find("div", {"class":"search-result-list news-search"}) # Note that this returns a list
	articles_indiv = articles_section.find_all('div', {'class': 'search-result-indiv'})

	for a in articles_indiv:
		try:
			headline = a.find('h3', {'class': 'search-result-title'}).text
			summary = a.find('div', {'class': 'search-result-excerpt'}).text
			link = a.find('a')['href']
			date = a.find('h5', {'class': 'search-result-timestamp'}).text

		except AttributeError:
			pass

		try:
			f.write(date.replace(",", ";") + "," + headline.replace(",", ";") + "," + "https://www.reuters.com"+link + "\n")
		except UnboundLocalError:
			pass

scraper(url)

#use requests to open first page
#read and copy csv
#open next page'''